package com.freez.spark.core.operation;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * FREEDOM  2021 人生苦短，不妨一试
 * FileName: MixedOperation.java
 * Author: zcs
 * Date: 2021年-12月-07日 周二 18:01
 * Description: 对filter,distinct,sample,glom算子进行演示
 */
public class MixedOperation {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf();
        conf.setAppName("SparkCore-MixedOperation").setMaster("local");
        JavaSparkContext jsc = new JavaSparkContext(conf);
        //数据
        List<Tuple2<String, Integer>> dataList = new ArrayList<>();
        dataList.add(new Tuple2<String, Integer>("2020-05-01", 11));
        dataList.add(new Tuple2<String, Integer>("2020-05-02", 4));
        dataList.add(new Tuple2<String, Integer>("2020-05-02", 7));
        dataList.add(new Tuple2<String, Integer>("2020-05-01", 4));
        dataList.add(new Tuple2<String, Integer>("2020-05-01", 10));
        JavaPairRDD<String, Integer> aRdd = jsc.parallelizePairs(dataList);
        JavaRDD<String> bRdd = jsc.parallelize(Arrays.asList("hello world", "hello hi"));
        /**
         * filter 过滤算子
         */
        //过滤掉“hello”
        JavaRDD<String> flatMapRDD = bRdd.flatMap((FlatMapFunction<String, String>) s -> Arrays.asList(s.split(" ")).iterator());
        JavaRDD<String> filterRDD = flatMapRDD.filter(new Function<String, Boolean>() {
            @Override
            public Boolean call(String s) throws Exception {
                return !"hello".equals(s);
            }
        });
        filterRDD.foreach((VoidFunction<String>) s -> System.out.println(s));
        /**
         * distinct 去重算子
         */
        JavaPairRDD<String, Integer> dRDD = jsc.parallelizePairs(Arrays.asList(
                new Tuple2<String, Integer>("2020-05-01", 1),
                new Tuple2<String, Integer>("2020-05-02", 2),
                new Tuple2<String, Integer>("2020-05-02", 2),
                new Tuple2<String, Integer>("2020-05-01", 4),
                new Tuple2<String, Integer>("2020-05-01", 10)
        ));
        JavaPairRDD<String, Integer> distinct = aRdd.distinct();
        distinct.foreach(s -> System.out.println(s));
        /**
         * sample 抽样算子
         */
        JavaPairRDD<String, Integer> sample = aRdd.sample(false, 0.5);
        sample.foreach((VoidFunction<Tuple2<String, Integer>>) s -> System.out.println(s));
        /**
         * glom
         */
        dRDD.glom().foreach(new VoidFunction<List<Tuple2<String, Integer>>>() {
            @Override
            public void call(List<Tuple2<String, Integer>> s) throws Exception {
                System.out.println(s);
            }
        });
        jsc.close();
    }
}
